'''
Created on 2011-12-28

@author: ajwang
'''
from urlparse import urlparse
from threading import Condition
from Queue import Queue
from threading import Thread
import time
from const import CONST


class UrlContainer(Thread):
    def __init__(self):
        Thread.__init__(self)
        self.daemon = True
        self.queue = Queue(1000)
        self.urls = {}
        self.files = {}
        self.num = 0
        self.start()
    
    def run(self):
        while True:
            _url = self.queue.get()
            try:
                self._add(_url)
            except:
                pass
            self.queue.task_done()
            #self.num += 1
            #print 'num: ' + str(self.num), ' - ', self.queue.unfinished_tasks
            
    def _add(self, _url):
        _d, _p = self.get_domain_path(_url)
        _d_l = self.urls.get(_d)
        
        if not _d_l:
            self.urls[_d] = {}
            self.files[_d] = open(CONST.DIR + _d, 'w', 0)
            _d_l = self.urls.get(_d)
            
        if not _d_l.get(_p):
            _d_l[_p] = 1
            self.files[_d].write(_d + _p + '\n')
            self.num += 1
            print 'num: ' + str(self.num)
    
    def get_domain_path(self, _url):
        _up = urlparse(_url)
        
        _path = _up.path
        if _up.query != '':
            _path  = _path + '?' + _up.query
        
        return [_up.netloc, _path]
    
    def add(self, _url):
        print 'add,', _url
        self.queue.put(_url)
        return _url
